x86/waitqueue: Because we have per-cpu stacks, we must wake up on teh

same cpu that we slept on. Otherwise stack references are bogus on
wakeup.

Signed-off-by: Keir Fraser <keir@xen.org>

diff --git a/xen/common/wait.c b/xen/common/wait.c
index b125dc5be2c98d8c46dee962a527b7fd67ba993a..d4feecb56ee38d94272418682aa944205b352469 100644 (file)
--- a/xen/common/wait.c
+++ b/xen/common/wait.c
@@ -34,6 +34,8 @@ struct waitqueue_vcpu {
      */
     void *esp;
     char *stack;
+    cpumask_t saved_affinity;
+    unsigned int wakeup_cpu;
 #endif
 };
 
@@ -106,9 +108,19 @@ void wake_up(struct waitqueue_head *wq)
 static void __prepare_to_wait(struct waitqueue_vcpu *wqv)
 {
     char *cpu_info = (char *)get_cpu_info();
+    struct vcpu *curr = current;
 
     ASSERT(wqv->esp == 0);
 
+    /* Save current VCPU affinity; force wakeup on *this* CPU only. */
+    wqv->wakeup_cpu = smp_processor_id();
+    cpumask_copy(&wqv->saved_affinity, curr->cpu_affinity);
+    if ( vcpu_set_affinity(curr, cpumask_of(wqv->wakeup_cpu)) )
+    {
+        gdprintk(XENLOG_ERR, "Unable to set vcpu affinity\n");
+        domain_crash_synchronous();
+    }
+
     asm volatile (
 #ifdef CONFIG_X86_64
         "push %%rax; push %%rbx; push %%rcx; push %%rdx; push %%rdi; "
@@ -144,6 +156,7 @@ static void __prepare_to_wait(struct waitqueue_vcpu *wqv)
 static void __finish_wait(struct waitqueue_vcpu *wqv)
 {
     wqv->esp = NULL;
+    (void)vcpu_set_affinity(current, &wqv->saved_affinity);
 }
 
 void check_wakeup_from_wait(void)
@@ -155,6 +168,20 @@ void check_wakeup_from_wait(void)
     if ( likely(wqv->esp == NULL) )
         return;
 
+    /* Check if we woke up on the wrong CPU. */
+    if ( unlikely(smp_processor_id() != wqv->wakeup_cpu) )
+    {
+        /* Re-set VCPU affinity and re-enter the scheduler. */
+        struct vcpu *curr = current;
+        cpumask_copy(&wqv->saved_affinity, curr->cpu_affinity);
+        if ( vcpu_set_affinity(curr, cpumask_of(wqv->wakeup_cpu)) )
+        {
+            gdprintk(XENLOG_ERR, "Unable to set vcpu affinity\n");
+            domain_crash_synchronous();
+        }
+        wait(); /* takes us back into the scheduler */
+    }
+
     asm volatile (
         "mov %1,%%"__OP"sp; rep movsb; jmp *(%%"__OP"sp)"
         : : "S" (wqv->stack), "D" (wqv->esp),